const fs = require('fs');
const http = require('http');
const https = require('https');
const cheerio = require('cheerio');
const httpUrl = require('url');
const request = require('request');

const {flie ,FileToRead,ImgesWrites} = require('./common/FileCreation')




class download {
    constructor(url: string) {
        this.NetworkImage(url)
    }
    NetworkImage(item: string) {
        const fileName = 'imgs';
        const filter = 'i';
        // let suffix = item.substr(item.length - 4);
        // let postfix = suffix.includes('.') ? item.substr(item.length - 4) : item.substr(item.length - 5);

        // console.log(item)

        ImgesWrites(fileName,item)
        // if (item.includes(filter) && (postfix === '.jpeg' || postfix === '.png' || postfix === '.gif' || postfix === '.jpg' || postfix === '.eps' || postfix === '.svg')) {
            // request(item).pipe(fs.createWriteStream(`${fileName}/${new Date().getTime()}${postfix}`));//写入图片到本
            // fs.appendFile('./dataCont/gather.txt',""+item+"\n",'utf8',function(err){  
            //     if(err)  {  
            //         console.log(err);  
            //     }  
            // });
            
        // }

    }
}

class touchfile {  //文件创建
    constructor(flie = null) {
        if (flie === null) {
            return;
        }
        const fe = flie.split('/');

        for (let i = 0; i < fe.length; i++) {
            switch (i) {
                case 0:
                    this.establish(`./${fe[0]}`);
                case 1:
                    this.establish(`./${fe[0]}/${fe[1]}`);
                case 2:
                    this.establish(`./${fe[0]}/${fe[1]}/${fe[2]}`);
            }
        }
    }
    establish(fileName: string) {


        fs.exists(fileName, (exists: any) => {  //文件夹创建
            if (exists) {
                console.log(`${fileName}文件存在`);
            } else {
                fs.mkdir(fileName, (err: any) => {
                    if (err) {
                        console.error(`创建文件：${fileName}失败！${err}`);
                    } else {
                        console.log(`创建文件：${fileName}成功！`);
                    }
                });
            }

        })
    }
}

/**
 * //图片提取到本地
 * @param {爬取的数据} ar 
 * @param {文件夹名称 默认为imgs} fileName 
 * @param {过滤图片} filter 
 * @param {过滤内容文件夹} catalogue 
 */

function extractive(ar: Array<any>, fileName: string, filter: string, catalogue: string) {

    fs.exists(fileName, (exists: any) => {
        if (exists === false) {
            if (fileName === 'imgs') {
                fs.mkdir(fileName, (err: any) => {
                    if (err) {
                        console.error(`创建文件：${fileName}失败！${err}`);
                    } else {
                        console.log(`创建文件：${fileName}成功！`);
                    }
                })

            } else if (fileName.split('/')) {

                new touchfile().establish(fileName);

            } else {

                new touchfile(fileName);
            }
        }
    })

setTimeout(() => {
    ar.forEach((item, index) => {
        let suffix = item.substr(item.length - 4);
        let postfix = suffix.includes('.') ? item.substr(item.length - 4) : item.substr(item.length - 5);
        if (catalogue) {
            if (item.includes(filter) && (postfix === '.jpeg' || postfix === '.png' || postfix === '.gif' || postfix === '.jpg' || postfix === '.eps' || postfix === '.svg') && item.includes(catalogue)) {
                // console.log(item)
                // request(item).pipe(fs.createWriteStream(`${fileName}/${new Date().getTime() + index}${postfix}`))

                fs.appendFile('./dataCont/gather.txt',""+item+"\n",'utf8',function(err){  
                    if(err)  {  
                        console.log(err);  
                    }  
                });
            }
        } else {
            if (item.includes(filter) && (postfix === '.jpeg' || postfix === '.png' || postfix === '.gif' || postfix === '.jpg' || postfix === '.eps' || postfix === '.svg')) {
                // console.log(item)
                // request(item).pipe(fs.createWriteStream(`${fileName}/${new Date().getTime() + index}${postfix}`))
                fs.appendFile('./dataCont/gather.txt',""+item+"\n",'utf8',function(err){  
                    if(err)  {  
                        console.log(err);  
                    }  
                });
            }
        }

    });
}, 1000);

}

function much(arrlist: any[]) {  //数组重复最多的值用来获取网址出来最多的域名
    const d = {}
    arrlist.forEach((k: string | number) => !d[k] ? d[k] = 1 : d[k]++)
    const max = Object.keys(d).sort((a, b) => d[b] - d[a])[0]
    return max
};

/**
 * 
 * @param {获取页面上所有的html元素} data 
 * @param {爬取的地址} url 
 * @param {爬取的标签 默认img} findleb 
 * @param {爬取的标签连接，默认src} Filelink 
 * @param {存储的文件夹名称} flieName 
 * @param {过滤文件夹名，保证内容图片} catalogue 
 */
function analysis(datacont: any) {  //数据提取
    const { data, url, findleb = 'img', Filelink = 'src', fileName, catalog, box } = datacont;
    const { statusCode } = data;
    const contentType = data.headers['content-type'];//文件类型
    let err = null;
    if (statusCode !== 200) {
        err = new Error('请求错误')
    } else if (!/^text\/html/.test(contentType)) {
        err = new ErrorEvent('文件类型错误')
    }

    if (err) {
        data.resume(); //重置缓存
        return false;
    }
    let htmltp = "";
    data.on('data', (chunk: { toString: (arg0: string) => string; }) => { //分段获取数据
        htmltp += chunk.toString('utf8');
    })


    data.on('end', () => { //数据获取完成

        // const bx = cheerio.load();
        let $ = null, flitarr = [];
        if (!!box) {
            console.log('有')
            const $1 = cheerio.load(htmltp);  //获取全面html
            $ = cheerio.load($1(box).html());
        } else {
            $ = cheerio.load(htmltp);
        }
        $(findleb).each((index: any, tiem: any) => {
            const hr = $(tiem).attr(Filelink);
            if (hr && (hr.includes('http') || hr.includes('https'))) {
                flitarr.push(hr);
            } else {
                if (hr && hr.startsWith('//')) {
                    const [, url] = hr.split("//");
                    flitarr.push(`http://${url}`)
                } else {
                    const { protocol, hostname } = httpUrl.parse(url);
                    flitarr.push(`${protocol}//${hostname}/${hr}`)
                }
            }
        })

        let analyzearr = [];
        flitarr.forEach((e, i) => {
            const { hostname } = httpUrl.parse(e);
            analyzearr.push(hostname)
        })
        const filter = much(analyzearr)
        // console.log(flitarr)
        extractive(flitarr, fileName, filter, catalog)
    })
}

function pachong(data: any) { //协议分支 ,和文件夹过滤
    const { url, catalog, box, fileName, Filelink } = data;
    if (url.includes('https')) {
        https.get(url, (res: any) => {
            analysis({
                data: res,
                url,
                catalog,
                box,
                fileName,
                Filelink
            })
        })
    } else {
        http.get(url, (res: any) => {
            analysis({
                data: res,
                url,
                catalog,
                box,
                fileName,
                Filelink
            })

        })
    }


}

//图片单页获取，可传入多页，为数组格式
function webSpider(data: { url?: any; catalogue: any; box: any; fileName?: "imgs"; Filelink?: "src"; }) {
    const { url = 'https://diyifangfuli.com/764.html?page=all', catalogue, box, fileName = 'imgs', Filelink = 'src' } = data
    if (typeof url === 'string') {
        pachong({ url, catalogue, box, fileName, Filelink })
    } else {
        url.forEach((url: any) => {
            pachong({ url, catalogue, box, fileName })
        })
    }
}

//图片多页获取
function webSpiders(data: { urlstr: any; urlend: any; fileName: any; find: any; link: any; catalogue: any; box: any; }, min: number, max: number, warn = 800) {

    const { urlstr, urlend, fileName = 'imgs', find, link, catalogue, box } = data;

    const interval = setInterval(() => {

        if (min === max) {
            clearInterval(interval);
        }
        min++;
        const url = `${urlstr}${min}${urlend}`;

        pachong({ url, catalogue, box, fileName })
    }, warn);
}

function DownloadAddress(url: string) {
    new download(url)
}



export { DownloadAddress, webSpiders, webSpider }